/*
 * This is the MIT license, see also http://www.opensource.org/licenses/mit-license.html
 *
 * Copyright (c) 2001 Brian Pitcher
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

// $Header: /cvsroot/weblech/weblech/src/weblech/spider/URLGetter.java,v 1.2 2002/06/02 08:02:45 weblech Exp $
package weblech.spider;

import java.io.*;
import java.net.Authenticator;
import java.net.HttpURLConnection;
import java.net.URL;

import org.apache.log4j.Logger;


/**请求远程URL,得到文件*/
public class URLGetter {
    private final static Logger logger = Logger.getLogger(URLGetter.class);


    /** 失败统计  */
    private int failureCount = 0;
    private final SpiderConfig config;

    public URLGetter(SpiderConfig config) {
        logger.debug("URLGetter()");
        this.config = config;

        Authenticator.setDefault(new DumbAuthenticator(
                config.getBasicAuthUser(), config.getBasicAuthPassword()));
    }

    /**
     * 请求下载url,读取文件内容
     * @param url
     * @return
     */
    public URLObject getURL(URLToDownload url) {
        if (url == null) {
            return null;
        }
        
        logger.info(Thread.currentThread().getName() + " : getURL ( " + url + " )");

        if (failureCount > 10) {
            logger.warn("Lots of failures recently, waiting 5 seconds before attempting download");
            
            try {
                Thread.sleep(5 * 1000);
            } catch (InterruptedException e) {}
            
            failureCount = 0;
        }

        URL requestURL = url.getURL();
        URL referer = url.getReferer();

        try {
            HttpURLConnection conn = (HttpURLConnection) requestURL.openConnection();

            if (referer != null) {
                logger.debug("Setting Referer header to " + referer);
                conn.setRequestProperty("Referer", referer.toExternalForm());
            }

            if (config.getUserAgent() != null) {
                logger.debug("Setting User-Agent to " + config.getUserAgent());
                conn.setRequestProperty("User-Agent", config.getUserAgent());
            }

            conn.setUseCaches(false);
            conn.setConnectTimeout(1000);
            logger.debug("Opening URL");

            long startTime = System.currentTimeMillis();
            conn.connect();

            String resp = conn.getResponseMessage();
            logger.debug("Remote server response: " + resp);
            
            int status = conn.getResponseCode();
            if (status != 200) {
                return null;
            }
            
            String respStr = conn.getHeaderField(0);
            logger.debug("Server response: " + respStr);

            for (int i = 1;; i++) {
                String key = conn.getHeaderFieldKey(i);

                if (key == null) {
                    break;
                }

                String value = conn.getHeaderField(key);
                logger.debug("Received header " + key + ": " + value);
            }

            logger.debug("Getting buffered input stream from remote connection");

            BufferedInputStream remoteBIS = new BufferedInputStream(conn.getInputStream()); //从远程地址得到
            ByteArrayOutputStream baos = new ByteArrayOutputStream(10240); 
            byte[] buf = new byte[1024]; //缓存
            int bytesRead = 0;

            while (bytesRead >= 0) {
                baos.write(buf, 0, bytesRead);
                bytesRead = remoteBIS.read(buf);
            }

            byte[] content = baos.toByteArray();
            long timeTaken = System.currentTimeMillis() - startTime;

            if (timeTaken < 100) {
                timeTaken = 500;
            }

            int bytesPerSec = (int) ((double) content.length / ((double) timeTaken / 1000.0));
            logger.debug("Downloaded " + content.length + " bytes, " + bytesPerSec + " bytes/sec");

            if (content.length < conn.getContentLength()) {
                logger.warn("Didn't download full content for URL: " + url);
                failureCount++;

                return null;
            }
            
            if (conn.getContentType() == null) {
                return null;
            }
            
            return new URLObject(requestURL, conn.getContentType(), content, config, content.length);
            
        } catch (FileNotFoundException fnfe) {
            logger.warn("File not found: " + fnfe.getMessage());

        } catch (IOException ioe) {
            logger.warn("Caught IO Exception: " + ioe.getMessage(), ioe);
            failureCount++;
        }
        
        return null;
    }
}
